{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b9583f02",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/xiongjiedai/llama.cpp\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/xiongjiedai/Library/Python/3.9/lib/python/site-packages/IPython/core/magics/osm.py:393: UserWarning: using bookmarks requires you to install the `pickleshare` library.\n",
      "  bkms = self.shell.db.get('bookmarks', {})\n",
      "/Users/xiongjiedai/Library/Python/3.9/lib/python/site-packages/IPython/core/magics/osm.py:417: UserWarning: using dhist requires you to install the `pickleshare` library.\n",
      "  self.shell.db['dhist'] = compress_dhist(dhist)[-100:]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Already up to date.\n"
     ]
    }
   ],
   "source": [
    "!git clone https://github.com/ggerganov/llama.cpp # Get the code for the first time use\n",
    "%cd ~/llama.cpp\n",
    "!git pull"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4f651103-a6c8-4b7d-9f0f-be0553b22acf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ggml-vocab-aquila.gguf             ggml-vocab-gpt2.gguf\n",
      "ggml-vocab-baichuan.gguf           ggml-vocab-llama-bpe.gguf\n",
      "ggml-vocab-bert-bge.gguf           ggml-vocab-llama-bpe.gguf.inp\n",
      "ggml-vocab-bert-bge.gguf.inp       ggml-vocab-llama-bpe.gguf.out\n",
      "ggml-vocab-bert-bge.gguf.out       ggml-vocab-llama-spm.gguf\n",
      "ggml-vocab-command-r.gguf          ggml-vocab-llama-spm.gguf.inp\n",
      "ggml-vocab-command-r.gguf.inp      ggml-vocab-llama-spm.gguf.out\n",
      "ggml-vocab-command-r.gguf.out      ggml-vocab-mpt.gguf\n",
      "ggml-vocab-deepseek-coder.gguf     ggml-vocab-mpt.gguf.inp\n",
      "ggml-vocab-deepseek-coder.gguf.inp ggml-vocab-mpt.gguf.out\n",
      "ggml-vocab-deepseek-coder.gguf.out ggml-vocab-phi-3.gguf\n",
      "ggml-vocab-deepseek-llm.gguf       ggml-vocab-phi-3.gguf.inp\n",
      "ggml-vocab-deepseek-llm.gguf.inp   ggml-vocab-phi-3.gguf.out\n",
      "ggml-vocab-deepseek-llm.gguf.out   ggml-vocab-refact.gguf\n",
      "ggml-vocab-falcon.gguf             ggml-vocab-refact.gguf.inp\n",
      "ggml-vocab-falcon.gguf.inp         ggml-vocab-refact.gguf.out\n",
      "ggml-vocab-falcon.gguf.out         ggml-vocab-stablelm.gguf\n",
      "ggml-vocab-gpt-2.gguf              ggml-vocab-starcoder.gguf\n",
      "ggml-vocab-gpt-2.gguf.inp          ggml-vocab-starcoder.gguf.inp\n",
      "ggml-vocab-gpt-2.gguf.out          ggml-vocab-starcoder.gguf.out\n",
      "ggml-vocab-gpt-neox.gguf\n"
     ]
    }
   ],
   "source": [
    "# Obtain the original LLaMA model weights and place them in ./models\n",
    "!ls ./models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cbaefb88",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/xiongjiedai/llama.cpp/models\n",
      "\u001b[34m==>\u001b[0m \u001b[1mDownloading https://formulae.brew.sh/api/formula.jws.json\u001b[0m\n",
      "\n",
      "\u001b[34m==>\u001b[0m \u001b[1mDownloading https://formulae.brew.sh/api/cask.jws.json\u001b[0m\n",
      "\n",
      "\u001b[34m==>\u001b[0m \u001b[1mDownloading https://ghcr.io/v2/homebrew/core/git-lfs/manifests/3.5.1\u001b[0m\n",
      "Already downloaded: /Users/xiongjiedai/Library/Caches/Homebrew/downloads/14375e477f0508d204fde94f6aed4c9774e693e3c90186a4577a570550a80a69--git-lfs-3.5.1.bottle_manifest.json\n",
      "\u001b[32m==>\u001b[0m \u001b[1mFetching \u001b[32mgit-lfs\u001b[39m\u001b[0m\n",
      "\u001b[34m==>\u001b[0m \u001b[1mDownloading https://ghcr.io/v2/homebrew/core/git-lfs/blobs/sha256:241e895f27\u001b[0m\n",
      "Already downloaded: /Users/xiongjiedai/Library/Caches/Homebrew/downloads/d7224901c11b06d6404dfa2acda8788addf0d5574247bfc5f3e9a96f90407272--git-lfs--3.5.1.arm64_ventura.bottle.tar.gz\n",
      "\u001b[34m==>\u001b[0m \u001b[1mPouring git-lfs--3.5.1.arm64_ventura.bottle.tar.gz\u001b[0m\n",
      "\u001b[34m==>\u001b[0m \u001b[1mCaveats\u001b[0m\n",
      "Update your git config to finish installation:\n",
      "\n",
      "  # Update global git config\n",
      "  $ git lfs install\n",
      "\n",
      "  # Update system git config\n",
      "  $ git lfs install --system\n",
      "\u001b[34m==>\u001b[0m \u001b[1mSummary\u001b[0m\n",
      "🍺  /opt/homebrew/Cellar/git-lfs/3.5.1: 78 files, 12.6MB\n",
      "\u001b[34m==>\u001b[0m \u001b[1mRunning `brew cleanup git-lfs`...\u001b[0m\n",
      "Disable this behaviour by setting HOMEBREW_NO_INSTALL_CLEANUP.\n",
      "Hide these hints with HOMEBREW_NO_ENV_HINTS (see `man brew`).\n",
      "Updated Git hooks.\n",
      "Git LFS initialized.\n",
      "Cloning into 'Llama-3-8B-GGUF'...\n",
      "remote: Enumerating objects: 44, done.\u001b[K\n",
      "remote: Counting objects: 100% (41/41), done.\u001b[K\n",
      "remote: Compressing objects: 100% (41/41), done.\u001b[K\n",
      "remote: Total 44 (delta 9), reused 0 (delta 0), pack-reused 3 (from 1)\u001b[K\n",
      "Unpacking objects: 100% (44/44), 2.25 MiB | 2.60 MiB/s, done.\n",
      "Filtering content: 100% (8/8), 9.46 GiB | 940.00 KiB/s, done.\n",
      "Cloning into 'Llama-3-70B-GGUF'...\n",
      "remote: Enumerating objects: 67, done.\u001b[K\n",
      "remote: Counting objects: 100% (64/64), done.\u001b[K\n",
      "remote: Compressing objects: 100% (64/64), done.\u001b[K\n",
      "remote: Total 67 (delta 4), reused 0 (delta 0), pack-reused 3 (from 1)\u001b[K\n",
      "Unpacking objects: 100% (67/67), 2.26 MiB | 3.37 MiB/s, done.\n",
      "Filtering content: 100% (40/40), 22.45 GiB | 882.00 KiB/s, done.\n"
     ]
    }
   ],
   "source": [
    "# Clone/Download the model files from Meta HF repo: https://huggingface.co/meta-llama. Or feel free to clone from my HF repo:\n",
    "%cd models\n",
    "!brew install git-lfs # Get git-lfs to clone large files\n",
    "!git lfs install\n",
    "!git clone https://huggingface.co/JaaackXD/Llama-3-8B-GGUF\n",
    "!git clone https://huggingface.co/JaaackXD/Llama-3-70B-GGUF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "12991b83",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/xiongjiedai/llama.cpp\n"
     ]
    }
   ],
   "source": [
    "!mkdir 8B-v3\n",
    "!mv Llama-3-8B-GGUF/*.gguf 8B-v3\n",
    "!mkdir 70B-v3\n",
    "!mv Llama-3-70B-GGUF/*.gguf 70B-v3\n",
    "%cd .."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "416a5513",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build\n",
    "!make clean && make -j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a57dd4e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Run the code if you would like to convert and quantize models by yourself\n",
    "# # Install Python dependencies\n",
    "# !python3 -m pip install -r requirements.txt\n",
    "\n",
    "# # Convert the HF models to ggml FP16 format (High RAM requirement!)\n",
    "# !python3 convert-hf-to-gguf.py models/Llama-3-8B-GGUF/Meta-Llama-3-8B/ --outfile models/8B-v3/ggml-model-f16.gguf --outtype f16\n",
    "# !python3 convert-hf-to-gguf.py models/Llama-3-70B-GGUF/Meta-Llama-3-70B/ --outfile models/70B-v3/ggml-model-f16.gguf --outtype f16\n",
    "\n",
    "# # Quantize the model to 4-bits (using Q4_K_M method)\n",
    "# !./quantize models/8B-v3/ggml-model-f16.gguf models/8B-v3/ggml-model-Q4_K_M.gguf Q4_K_M\n",
    "# !./quantize models/70B-v3/ggml-model-f16.gguf models/70B-v3/ggml-model-Q4_K_M.gguf Q4_K_M"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bd5c3a00",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 2797 (858f6b73)\n",
      "main: built with Apple clang version 14.0.3 (clang-1403.0.22.14.1) for arm64-apple-darwin22.1.0\n",
      "main: seed  = 0\n",
      "llama_model_loader: loaded meta data with 21 key-value pairs and 291 tensors from ./models/8B-v3/ggml-model-Q4_K_M.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = Meta-Llama-3-8B\n",
      "llama_model_loader: - kv   2:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   3:                       llama.context_length u32              = 8192\n",
      "llama_model_loader: - kv   4:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 14336\n",
      "llama_model_loader: - kv   6:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   7:              llama.attention.head_count_kv u32              = 8\n",
      "llama_model_loader: - kv   8:                       llama.rope.freq_base f32              = 500000.000000\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 15\n",
      "llama_model_loader: - kv  11:                           llama.vocab_size u32              = 128256\n",
      "llama_model_loader: - kv  12:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv  13:                       tokenizer.ggml.model str              = gpt2\n",
      "llama_model_loader: - kv  14:                         tokenizer.ggml.pre str              = llama-bpe\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.tokens arr[str,128256]  = [\"!\", \"\\\"\", \"#\", \"$\", \"%\", \"&\", \"'\", ...\n",
      "llama_model_loader: - kv  16:                  tokenizer.ggml.token_type arr[i32,128256]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...\n",
      "llama_model_loader: - kv  17:                      tokenizer.ggml.merges arr[str,280147]  = [\"Ġ Ġ\", \"Ġ ĠĠĠ\", \"ĠĠ ĠĠ\", \"...\n",
      "llama_model_loader: - kv  18:                tokenizer.ggml.bos_token_id u32              = 128000\n",
      "llama_model_loader: - kv  19:                tokenizer.ggml.eos_token_id u32              = 128001\n",
      "llama_model_loader: - kv  20:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q4_K:  193 tensors\n",
      "llama_model_loader: - type q6_K:   33 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 256/128256 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = BPE\n",
      "llm_load_print_meta: n_vocab          = 128256\n",
      "llm_load_print_meta: n_merges         = 280147\n",
      "llm_load_print_meta: n_ctx_train      = 8192\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 8\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_embd_head_k    = 128\n",
      "llm_load_print_meta: n_embd_head_v    = 128\n",
      "llm_load_print_meta: n_gqa            = 4\n",
      "llm_load_print_meta: n_embd_k_gqa     = 1024\n",
      "llm_load_print_meta: n_embd_v_gqa     = 1024\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: f_logit_scale    = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 14336\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: causal attn      = 1\n",
      "llm_load_print_meta: pooling type     = 0\n",
      "llm_load_print_meta: rope type        = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 500000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 8192\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: ssm_d_conv       = 0\n",
      "llm_load_print_meta: ssm_d_inner      = 0\n",
      "llm_load_print_meta: ssm_d_state      = 0\n",
      "llm_load_print_meta: ssm_dt_rank      = 0\n",
      "llm_load_print_meta: model type       = 8B\n",
      "llm_load_print_meta: model ftype      = Q4_K - Medium\n",
      "llm_load_print_meta: model params     = 8.03 B\n",
      "llm_load_print_meta: model size       = 4.58 GiB (4.89 BPW) \n",
      "llm_load_print_meta: general.name     = Meta-Llama-3-8B\n",
      "llm_load_print_meta: BOS token        = 128000 '<|begin_of_text|>'\n",
      "llm_load_print_meta: EOS token        = 128001 '<|end_of_text|>'\n",
      "llm_load_print_meta: LF token         = 128 'Ä'\n",
      "llm_load_print_meta: EOT token        = 128009 '<|eot_id|>'\n",
      "llm_load_tensors: ggml ctx size =    0.30 MiB\n",
      "ggml_backend_metal_log_allocated_size: allocated buffer, size =  4403.50 MiB, ( 4403.56 / 49152.00)\n",
      "llm_load_tensors: offloading 32 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 33/33 layers to GPU\n",
      "llm_load_tensors:      Metal buffer size =  4403.50 MiB\n",
      "llm_load_tensors:        CPU buffer size =   281.81 MiB\n",
      ".......................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 8192\n",
      "llama_new_context_with_model: n_batch    = 2048\n",
      "llama_new_context_with_model: n_ubatch   = 512\n",
      "llama_new_context_with_model: flash_attn = 0\n",
      "llama_new_context_with_model: freq_base  = 500000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "ggml_metal_init: allocating\n",
      "ggml_metal_init: found device: Apple M1 Max\n",
      "ggml_metal_init: picking default device: Apple M1 Max\n",
      "ggml_metal_init: default.metallib not found, loading from source\n",
      "ggml_metal_init: GGML_METAL_PATH_RESOURCES = nil\n",
      "ggml_metal_init: loading '/Users/xiongjiedai/llama.cpp/ggml-metal.metal'\n",
      "ggml_metal_init: GPU name:   Apple M1 Max\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyApple7  (1007)\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyCommon3 (3003)\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyMetal3  (5001)\n",
      "ggml_metal_init: simdgroup reduction support   = true\n",
      "ggml_metal_init: simdgroup matrix mul. support = true\n",
      "ggml_metal_init: hasUnifiedMemory              = true\n",
      "ggml_metal_init: recommendedMaxWorkingSetSize  = 51539.61 MB\n",
      "llama_kv_cache_init:      Metal KV buffer size =  1024.00 MiB\n",
      "llama_new_context_with_model: KV self size  = 1024.00 MiB, K (f16):  512.00 MiB, V (f16):  512.00 MiB\n",
      "llama_new_context_with_model:        CPU  output buffer size =     0.49 MiB\n",
      "llama_new_context_with_model:      Metal compute buffer size =   560.00 MiB\n",
      "llama_new_context_with_model:        CPU compute buffer size =    24.01 MiB\n",
      "llama_new_context_with_model: graph nodes  = 1030\n",
      "llama_new_context_with_model: graph splits = 2\n",
      "\n",
      "system_info: n_threads = 8 / 10 | AVX = 0 | AVX_VNNI = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | MATMUL_INT8 = 0 | LLAMAFILE = 1 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temperature \n",
      "generate: n_ctx = 8192, n_batch = 2048, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m<|begin_of_text|> First Citizen:\n",
      "\n",
      " Before we proceed any further, hear me speak.\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " Speak, speak.\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " You are all resolved rather to die than to famish?\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " Resolved. resolved.\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " First, you know Caius Marcius is chief enemy to the people.\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " We know't, we know't.\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " Let us kill him, and we'll have corn at our own price. Is't a verdict?\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " No more talking on't; let it be done: away, away!\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " One word, good citizens.\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " We are accounted poor citizens, the patricians good. What authority surfeits on would relieve us: if they would yield us but the superfluity,  while it were wholesome, we might guess they relieved us humanely; but they think we are too dear: the leanness that afflicts us, the object of  our misery, is as an inventory to particularise their abundance; our sufferance is a gain to them Let us revenge this with our pikes,  ere we become rakes: for the gods know I speak this in hunger for bread, not in thirst for revenge.\n",
      "\n",
      " \n",
      "\n",
      " \u001b[0m First Citizen:\n",
      "\n",
      " Do't when you will: 'tis like they'll never lift it.\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " Why should they then pretend to justice? Come, I trifle thus with them in havior of my wrath; now turn like spiders in my chest; the fury panteth. Ha! There goes the matter on my soul  --upon my head! Let not your eyes I prithe yield unto my tears; hold them back! Here they come: serve but for laughter, and yet farewel!\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " Are these your shrewd men of the city.\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " Ha!\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " What are their debts? what cannot be done to help them?\n",
      "\n",
      " \n",
      "\n",
      " Third Citizen:\n",
      "\n",
      " Knives will not pluck up grasse; tis no time in anger to  use such wrough devices: lend, lend.\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " This is mere prating; and the consequences are Like stormy weather at sea. When good friends, say so\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " Let them pull down Caesar's statues and destroy His altars: Melt away the gulfs, which those superfluous shewrs Which she o'erspreads? Oth'rough-powdered infamy! Quench'd in the fire of force, let all be spent, What is't to pray to anough! what, to beseeech Him\n",
      "\n",
      " \n",
      "\n",
      " Third Citizen:\n",
      "\n",
      " Come.\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " The games!\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " Be it as some may. The fences may be reard To keep the creatures in: yet 'tis a course  Preposterously conceived, in means too weak And things far past all use.\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " Rescue thyself; whilst thou mayst have no need.\n",
      "\n",
      " \n",
      "\n",
      " Third Citizen:\n",
      "\n",
      " See they breathe! they move! but hear 'em not hark!\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " Peace! peace!\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " We'll overhear them.  --A plentifull city, I think was this Rome: as, to shine for prescience; take it for mine, A beseeming ceremony of late days; Here's Buckingham importun'd, Caes'ars uncle.  See how our Jove defies the light! Nay, but hear me.\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " Pray you, let's go see them.  --Is Caesar sick?  we have heard, oh 'tis a quarrel: Sound forth nobility. Come; see that here Is spread for these but stout men.\n",
      "\n",
      " \n",
      "\n",
      " Third Citizen:\n",
      "\n",
      " Here's the way to lay 'em in the field still more proudly than they are awake for.\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " Good faith this light pretends with him in the Cabalent, he hath rid their discretions being but rude and wanton.  --Away, away!\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " Come on; it seems some excellent nectar have they tasted, and that have prov'd their vices common common to those of their houses.\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " Here's goodly lordship: Prythee no more prating!  Enter BRUTUS\n",
      "\n",
      " \n",
      "\n",
      " Brutus:\n",
      "\n",
      " Stand close, my Lord and let us for a while turn our silent eyes from Mars to these Shepherds that would follow him unto the field; where by Jove's will we shall show ourselves as we are.\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " They do not go deep enough, they know not what  is to incurr himselfe a murder; for Caesar must be kill'd before we can begin afresh. Our course will seem too bloody, shoulde he die cutt him out in little love-pieces and then devour him. I think no man worth another's study: And yet by Jove, I have an over-awing spirit, that does at onetime rid his nature of his faith so that I did never dream on't.\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " You say you are a heathen, will you not be christen'd and have true religion amongst us?\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " The fire from hence is easily blown to other men: if it cannot be kill'd until I do expire; I will kill 'tis necessary.\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " Do so, I'll this belief of yours. If Caesar get him a new fortune, he shall not love the people the more for Caesars brother's death, than that the folks of Rome are any thing jealous of it: the one and fiftie in his heart is seated, and like to root out many a three-pearced tyrant.\n",
      "\n",
      " \n",
      "\n",
      " Brutus:\n",
      "\n",
      " Peace! do not speak to me, e'e; enuff's enough: It hath made me almost stiffe: Come gentlemen, be patient: altho Caesar may have more capacities and qualities than a weasell hath, it will be hid under a howlet's visage, never dare I draw my sword.\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " But gentlemen our country, sickness will not admit the med'cine but of noble truthe. It is not yet so\n",
      "llama_print_timings:        load time =    9853.69 ms\n",
      "llama_print_timings:      sample time =     269.45 ms /  1024 runs   (    0.26 ms per token,  3800.33 tokens per second)\n",
      "llama_print_timings: prompt eval time =     668.64 ms /   258 tokens (    2.59 ms per token,   385.86 tokens per second)\n",
      "llama_print_timings:        eval time =   27287.47 ms /  1023 runs   (   26.67 ms per token,    37.49 tokens per second)\n",
      "llama_print_timings:       total time =   28614.72 ms /  1281 tokens\n",
      "ggml_metal_free: deallocating\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "# Start inference on a gguf model (-h to show all options)\n",
    "!./main -ngl 10000 -m ./models/8B-v3/ggml-model-Q4_K_M.gguf --color --temp 1.1 --repeat_penalty 1.1 -c 0 -n 1024 -e -s 0 -p \"\"\"\\\n",
    "First Citizen:\\n\\n\\\n",
    "Before we proceed any further, hear me speak.\\n\\n\\\n",
    "\\n\\n\\\n",
    "All:\\n\\n\\\n",
    "Speak, speak.\\n\\n\\\n",
    "\\n\\n\\\n",
    "First Citizen:\\n\\n\\\n",
    "You are all resolved rather to die than to famish?\\n\\n\\\n",
    "\\n\\n\\\n",
    "All:\\n\\n\\\n",
    "Resolved. resolved.\\n\\n\\\n",
    "\\n\\n\\\n",
    "First Citizen:\\n\\n\\\n",
    "First, you know Caius Marcius is chief enemy to the people.\\n\\n\\\n",
    "\\n\\n\\\n",
    "All:\\n\\n\\\n",
    "We know't, we know't.\\n\\n\\\n",
    "\\n\\n\\\n",
    "First Citizen:\\n\\n\\\n",
    "Let us kill him, and we'll have corn at our own price. Is't a verdict?\\n\\n\\\n",
    "\\n\\n\\\n",
    "All:\\n\\n\\\n",
    "No more talking on't; let it be done: away, away!\\n\\n\\\n",
    "\\n\\n\\\n",
    "Second Citizen:\\n\\n\\\n",
    "One word, good citizens.\\n\\n\\\n",
    "\\n\\n\\\n",
    "First Citizen:\\n\\n\\\n",
    "We are accounted poor citizens, the patricians good. What authority surfeits on would relieve us: if they would yield us but the superfluity, \\\n",
    "while it were wholesome, we might guess they relieved us humanely; but they think we are too dear: the leanness that afflicts us, the object of \\\n",
    "our misery, is as an inventory to particularise their abundance; our sufferance is a gain to them Let us revenge this with our pikes, \\\n",
    "ere we become rakes: for the gods know I speak this in hunger for bread, not in thirst for revenge.\\n\\n\\\n",
    "\\n\\n\\\n",
    "\"\"\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17c21608",
   "metadata": {},
   "source": [
    "## Benchmarks"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b6aee37",
   "metadata": {},
   "source": [
    "### 8B Q4_K_M"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "25e6834b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| model                          |       size |     params | backend    | ngl | test       |              t/s |\n",
      "| ------------------------------ | ---------: | ---------: | ---------- | --: | ---------- | ---------------: |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | Metal      |  99 | pp 512     |   408.23 ± 24.57 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | Metal      |  99 | pp 1024    |   355.45 ± 12.93 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | Metal      |  99 | pp 2048    |    347.96 ± 1.37 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | Metal      |  99 | pp 4096    |    329.84 ± 1.02 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | Metal      |  99 | pp 8192    |    302.92 ± 0.83 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | Metal      |  99 | tg 512     |     35.73 ± 0.27 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | Metal      |  99 | tg 1024    |     34.49 ± 0.08 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | Metal      |  99 | tg 2048    |     33.60 ± 0.29 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | Metal      |  99 | tg 4096    |     31.18 ± 0.02 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | Metal      |  99 | tg 8192    |     26.84 ± 0.02 |\n",
      "\n",
      "build: 858f6b73 (2797)\n"
     ]
    }
   ],
   "source": [
    "!./llama-bench -p 512,1024,2048,4096,8192 -n 512,1024,2048,4096,8192 -m ./models/8B-v3/ggml-model-Q4_K_M.gguf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f07d6cd",
   "metadata": {},
   "source": [
    "### 8B F16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e8d4d00c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| model                          |       size |     params | backend    | ngl | test       |              t/s |\n",
      "| ------------------------------ | ---------: | ---------: | ---------- | --: | ---------- | ---------------: |\n",
      "| llama 8B F16                   |  14.96 GiB |     8.03 B | Metal      |  99 | pp 512     |    517.34 ± 5.74 |\n",
      "| llama 8B F16                   |  14.96 GiB |     8.03 B | Metal      |  99 | pp 1024    |   418.77 ± 28.34 |\n",
      "| llama 8B F16                   |  14.96 GiB |     8.03 B | Metal      |  99 | pp 2048    |   400.44 ± 13.72 |\n",
      "| llama 8B F16                   |  14.96 GiB |     8.03 B | Metal      |  99 | pp 4096    |   374.09 ± 10.61 |\n",
      "| llama 8B F16                   |  14.96 GiB |     8.03 B | Metal      |  99 | pp 8192    |    351.46 ± 1.28 |\n",
      "| llama 8B F16                   |  14.96 GiB |     8.03 B | Metal      |  99 | tg 512     |     18.75 ± 0.05 |\n",
      "| llama 8B F16                   |  14.96 GiB |     8.03 B | Metal      |  99 | tg 1024    |     18.43 ± 0.10 |\n",
      "| llama 8B F16                   |  14.96 GiB |     8.03 B | Metal      |  99 | tg 2048    |     17.37 ± 0.20 |\n",
      "| llama 8B F16                   |  14.96 GiB |     8.03 B | Metal      |  99 | tg 4096    |     16.33 ± 0.04 |\n",
      "| llama 8B F16                   |  14.96 GiB |     8.03 B | Metal      |  99 | tg 8192    |     15.03 ± 0.09 |\n",
      "\n",
      "build: 858f6b73 (2797)\n"
     ]
    }
   ],
   "source": [
    "!./llama-bench -p 512,1024,2048,4096,8192 -n 512,1024,2048,4096,8192 -m ./models/8B-v3/ggml-model-f16.gguf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72d34884",
   "metadata": {},
   "source": [
    "### 70B Q4_K_M"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b704364b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| model                          |       size |     params | backend    | ngl | test       |              t/s |\n",
      "| ------------------------------ | ---------: | ---------: | ---------- | --: | ---------- | ---------------: |\n",
      "| llama 70B Q4_K - Medium        |  39.59 GiB |    70.55 B | Metal      |  99 | pp 512     |     34.96 ± 0.79 |\n",
      "| llama 70B Q4_K - Medium        |  39.59 GiB |    70.55 B | Metal      |  99 | pp 1024    |     33.01 ± 1.05 |\n",
      "| llama 70B Q4_K - Medium        |  39.59 GiB |    70.55 B | Metal      |  99 | pp 2048    |     33.21 ± 0.24 |\n",
      "| llama 70B Q4_K - Medium        |  39.59 GiB |    70.55 B | Metal      |  99 | pp 4096    |     32.64 ± 0.33 |\n",
      "| llama 70B Q4_K - Medium        |  39.59 GiB |    70.55 B | Metal      |  99 | pp 8192    |     30.97 ± 0.09 |\n",
      "| llama 70B Q4_K - Medium        |  39.59 GiB |    70.55 B | Metal      |  99 | tg 512     |      4.34 ± 0.08 |\n",
      "| llama 70B Q4_K - Medium        |  39.59 GiB |    70.55 B | Metal      |  99 | tg 1024    |      4.09 ± 0.02 |\n",
      "| llama 70B Q4_K - Medium        |  39.59 GiB |    70.55 B | Metal      |  99 | tg 2048    |      4.12 ± 0.14 |\n",
      "| llama 70B Q4_K - Medium        |  39.59 GiB |    70.55 B | Metal      |  99 | tg 4096    |      4.09 ± 0.00 |\n",
      "| llama 70B Q4_K - Medium        |  39.59 GiB |    70.55 B | Metal      |  99 | tg 8192    |      3.71 ± 0.01 |\n",
      "\n",
      "build: 858f6b73 (2797)\n"
     ]
    }
   ],
   "source": [
    "!./llama-bench -p 512,1024,2048,4096,8192 -n 512,1024,2048,4096,8192 -m ./models/70B-v3/ggml-model-Q4_K_M.gguf"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
